# -*- coding: utf-8 -*-

#import
import os
import string
from time import strftime, gmtime
import sqlite3 as lite


#define
download_dir = "./tmp/" #path del file
file_name_domains = "domains.txt"
n_lines_ignore = 5 #numero di righe da ignorare
#file_name_justdomains = "justdomains.txt"
name_db = "domains_v1.0.db" #nome del db
todate = strftime("%Y-%m-%d %H:%M:%S", gmtime()) #stringa della data di attivazione dello script
file_name_bulk = "bulk_registrars.txt"
file_name_dynamicdns = "dynamic_dns.txt"
file_name_freewebhosts ="freewebhosts.txt"
file_name_immortal = "immortal_domains.txt"

#sqlite
con = lite.connect(name_db)
con.text_factory = str
with con:
    cur = con.cursor()    
    cur.execute("CREATE TABLE IF NOT EXISTS domains(id INTEGER PRIMARY KEY, domain TEXT, categoria TEXT, data TEXT, fonte TEXT, reputazione TEXT, path TEXT)")


#creazione della directory se non esiste
if not os.path.exists(download_dir):
    os.makedirs(download_dir)


#os.system("wget -q -Y on 'http://malwaredomains.lehigh.edu/files/justdomains' -O " + download_dir + file_name_justdomains)

#f = open(download_dir + file_name_justdomains)

#while True:
        #line = f.readline()
        #if not line: break
        #line = line[:-1]
        #print "justdomain:", line #stampa di test
        #inserimento nel db
        #cur.execute("INSERT INTO domains(domain,categoria,data,fonte,reputazione) VALUES (?,?,?,?,?)",
                    #(line, "malware", todate, "_malwaredomains_", "black list"))
       
       
#con.commit()
#con.close()      
#f.close()
#os.remove(download_dir + file_name_justdomains)

  
os.system("wget -q -Y on 'http://malwaredomains.lehigh.edu/files/domains.txt' -O " + download_dir + file_name_domains)

f = open(download_dir + file_name_domains)

while n_lines_ignore > 0:
    line = f.readline()
    n_lines_ignore -= 1

c = 0
domain = categoria = fonte = ""
while True:
    line = f.readline()
    if not line: break
    if line[0] == "#":
        #print line[:-1]
        continue
    #print line #stampa di test
    lista = string.split(line,"\t")
    n_elem = 1
    #print "contatore:", c
    for elem in lista:
        if n_elem == 3:
            domain = elem
            #print "domain:", elem #stampa di test
        if n_elem == 4:
            categoria = elem
            #print "categoria:", elem #stampa di test
        if n_elem == 5:
            #print "fonte:", elem, "(_malware domains_)" #stampa di test
            fonte = elem + " (_malware domains_)"
        n_elem += 1
    cur.execute("INSERT INTO domains(domain,categoria,data,fonte,reputazione) VALUES (?,?,?,?,?)",
                    (domain, categoria, todate, fonte, "black list"))
    #print domain, "\t", categoria, "\t", todate, "\t", fonte #stampa di test
    #cur.execute("UPDATE domains SET categoria=?, fonte=? WHERE domain=?",
                    #("malware,"+categoria,fonte,domain))
    c += 1
#print c  #stampa di test
con.commit()
#con.close()
f.close()
os.remove(download_dir + file_name_domains)





#bulk_registrars.txt -> blacklist
n_lines_ignore = 3 #numero di righe da ignorare

os.system("wget -q -Y on 'http://malwaredomains.lehigh.edu/files/bulk_registrars.txt' -O " + download_dir + file_name_bulk)

f = open(download_dir + file_name_bulk)

while n_lines_ignore > 0:
    line = f.readline()
    n_lines_ignore -= 1

prec = ""
while True:
    line = f.readline()
    if not line: break
    if line[0] == "#":
        #print line[:-1]
        continue
    line = line[2:-2]
    #print "bulk_registrars:", line #stampa di test
    lista = string.split(line,"\t")
    n_elem = 1
    for elem in lista:
        if n_elem == 1:
            domain = elem
            #print "domain:", elem #stampa di test
        n_elem += 1
    if not prec == domain:
        #inserimento nel db
        cur.execute("INSERT INTO domains(domain,categoria,data,fonte,reputazione) VALUES (?,?,?,?,?)",
                    (domain, "probably domain abused", todate, "_malwaredomains_", "black list"))
    prec = domain
       
con.commit()
#con.close()      
f.close()
os.remove(download_dir + file_name_bulk)




#dynamic_dns.txt -> white list
n_lines_ignore = 14 #numero di righe da ignorare

os.system("wget -q -Y on 'http://malwaredomains.lehigh.edu/files/dynamic_dns.txt' -O " + download_dir + file_name_dynamicdns)

f = open(download_dir + file_name_dynamicdns)

while n_lines_ignore > 0:
    line = f.readline()
    n_lines_ignore -= 1

prec = ""
while True:
    line = f.readline()
    if not line: break
    #print line[:-1]
    if line[0] == "#":
        #print line[:-1]
        continue
    lista = string.split(line,"#")
    n_elem = 1
    for elem in lista:
        if n_elem == 1:
            lista_b = string.split(elem,"\t")
            n_elem_b = 1
            for elem_b in lista_b:
                if n_elem_b == 1:
                    #print "dynamic_dns:", elem_b #stampa di test
                    if not prec == elem_b:
                        #print "dynamic_dns:", elem_b #stampa di test
                        #inserimento nel db
                        cur.execute("INSERT INTO domains(domain,categoria,data,fonte,reputazione) VALUES (?,?,?,?,?)",
                                    (elem_b, "dynamic dns provider", todate, "_malwaredomains_", "white list"))
                    prec = elem_b
                n_elem_b += 1
        n_elem += 1
   
       
con.commit()
#con.close()      
f.close()
os.remove(download_dir + file_name_dynamicdns)


#dynamic_dns.txt -> white list
# non funziona: nelle righe che non hanno # da problemi nel db

n_lines_ignore = 3 #numero di righe da ignorare
os.system("wget -q -Y on 'http://malwaredomains.lehigh.edu/files/freewebhosts.txt' -O " + download_dir + file_name_freewebhosts)

f = open(download_dir + file_name_freewebhosts)

while n_lines_ignore > 0:
    line = f.readline()
    n_lines_ignore -= 1
    
while True:
    line = f.readline()
    if not line: break
    if line[0] == "#": #al momento inutile 
        #print line[:-1] #stampa di test
        continue
    line = line[:-1]
    #print "freewebhosts:", line #stampa di test
    lista = string.split(line,"#")
    n_elem = 1
    for elem in lista:
        if n_elem == 1:
            domain = elem
            #print "domain:", elem #stampa di test
        n_elem += 1
    #inserimento nel db
    #print "domain:", domain #stampa di test
    cur.execute("INSERT INTO domains(domain,categoria,data,fonte,reputazione) VALUES (?,?,?,?,?)",
                (domain, "free web hosts", todate, "_malwaredomains_", "white list"))
   
       
con.commit()
#con.close()      
f.close()
os.remove(download_dir + file_name_freewebhosts)



#immortal_domains.txt -> black list
os.system("wget -q -Y on 'http://malwaredomains.lehigh.edu/files/immortal_domains.txt' -O " + download_dir + file_name_immortal)

f = open(download_dir + file_name_immortal)

while True:
        line = f.readline()
        if not line: break
        if line[0] == "#": 
            #print line[:-1] #stampa di test
            continue
        line = line[:-1]
        #print "immortal domain:", line #stampa di test
        #inserimento nel db
        cur.execute("INSERT INTO domains(domain,categoria,data,fonte,reputazione) VALUES (?,?,?,?,?)",
                    (line, "long-lived malware domains", todate, "_malwaredomains_", "black list"))
       
       
con.commit()
con.close()      
f.close()
os.remove(download_dir + file_name_immortal)
